from urllib.parse import unquote

import aiohttp
import requests
from lxml import etree
import asyncio



async def fetch(session,url):
    async with session.get(url) as poetry_list:
        print(f'开始爬取：{url}')
        text = await poetry_list.text()
        poetry_html = etree.HTML(text)
        poetry = poetry_html.xpath("//div[@class='left']//div[@class='sons']/div")

        for j in poetry:
            title = j.xpath("./a")[0].text
            print("诗：", title, )


        next_url = poetry_html.xpath("//a[@class='amore']")[0].get('href')
        # print(next_url)
        if next_url:
            next_url = 'https://www.gushiwen.cn/' + next_url
            url = next_url
            await fetch(session,url)
        else:
            print("没有下一页")


async def main():
    async with aiohttp.ClientSession() as session:
        async with session.get('https://www.gushiwen.cn/mingjus') as response:
            text = await response.text()
            html = etree.HTML(text)
            poemers = html.xpath("//div[@id='type2']/div[@class='sright']/a")

            author_task = [fetch(session,f'https://www.gushiwen.cn{a.xpath("./@href")[0]}') for a in poemers]

            await asyncio.gather(*author_task)

if __name__ == '__main__':
    asyncio.run(main())


#
# url0 = 'https://www.gushiwen.cn/mingjus'
# response = requests.get(url0)
# html = etree.HTML(response.text)
# poemers = html.xpath("//div[@id='type2']/div[@class='sright']/a")
# for i in poemers:
#     page = 0
#     poem = i.text  # 标签内文本
#     href = i.get('href')
#     href = unquote(href)
#     print(poem,href)
#     print(f'下面是{poem}的名句')
#     url = f'https://www.gushiwen.cn{href}'
#     while True:
#         print(f'正在爬取{poem}系列的第{page}页')
#         poetry_list = requests.get(url)
#         poetry_html = etree.HTML(poetry_list.text)
#         poetry = poetry_html.xpath("//div[@class='left']//div[@class='sons']/div")
#         for j in poetry:
#             title = j.xpath("./a")[0].text
#             source = j.xpath("./a")[1].text
#             print("诗：",title,'来源：',source)
#         next_url = poetry_html.xpath("//a[@class='amore']")[0].get('href')
#
#         # print(next_url)
#         if next_url:
#             next_url = 'https://www.gushiwen.cn/' + next_url
#             url = next_url
#             page += 1
#         else:
#             break
#
#














